#!/usr/bin/env python
#coding=utf-8
import os, sys
import re
import datetime
import ben_minidom 
from stat_conf import topwords, important_domain, eth_device
from stat_conf import file_size, is_title, top_n
from stat_conf import time_interval, is_file_split

def str2restr(l):
    reli = []
    for s in l:
        s = s + '$'
        w = s.split('.')
        if (w[0] == '*'):
            w[0] = '.*'
        reli.append('\.'.join(w))
    return reli

def getline():
    while True:
        try:
            line = raw_input()
        except EOFError, e:
            sys.exit(0)

        if len(line) == 0: continue
        if line[0:6] == "tail: ": continue
        if line[0] != '#':
            return line

def get_new_time(dl, minute):
    objt = datetime.datetime.strptime(dl, "%Y%m%d%H%M%S")
    #delta = datetime.timedelta(minutes=minute)
    delta = datetime.timedelta(seconds=minute)
    objt += delta
    return objt.strftime("%Y%m%d%H%M%S")

sum_barrier = ["start-time", "end-time", "serverip", "succ_user", "succ_system", "fail-num", "recursion", "non-recursi", "total-query", "frequency", "succ_sys%", "succ_user%", "delay-avg"]
sum_barrier_cn = [u"开始时间", u"结束时间", u"DNS服务器IP", u"用户解析成功数", u"系统解析成功数", u"失败数", u"递归请求查询数", u"非递归查询>数", u"总请求数", u"查询频率(次/秒)", u"系统解析成功率(%)", u"用户解析成功率(%)", u"解析平均时延(ms)"]

top_barrier = ["starttime", "endtime", "serverip", "domain-name", "request", "total", "ratio", "success%"]
top_barrier_cn = [u"统计开始时间",u"统计结束时间", u"DNS服务器IP", u"域名", u"域名解析请求次数", u"总请求次数", u"比例", u"解析成功率"]

imp_barrier = ["starttime", "endtime", "serverip", "domain-name", "total", "succ", "fail", "freq", "su%", "delay"]
imp_barrier_cn = [u"开始时间", u"结束时间", u"DNS服务器IP", u"解析域名", u"域名解析总数", u"域名解析成功数", u"失败数", u"查询频率(次/秒)", u"系统解析成功率(%)", u"解析平均时延(ms)"]

file_no = {"summary":0, "imp":0, "top":0}
def gen_doc_and_class(zhEN = 'AC', zhCN = 'AC', xsi = 'xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance'):
    doc = ben_minidom.Document()
    class_elem = doc.createElement("class")
    class_elem.setAttribute("zhEN", zhEN)
    class_elem.setAttribute("zhCN", zhCN)
    res_elem = doc.createElement("resource-data")
    res_elem.setAttribute("xmlns:xsi", xsi)
    res_elem.setAttribute("xsi:noNamespaceSchemaLocation","resource-data.xsd")
    res_elem.appendChild(class_elem)
    doc.appendChild(res_elem)
    return (doc, class_elem)
def add_attr_value(doc, elem, attr_list):
    attr_elems= doc.createElement("attributes")
    elem.appendChild(attr_elems)
    for attr in attr_list:
        node = doc.createElement("attribute")
        node.setAttribute("columnEN", attr["en"])
        node.setAttribute("columnCN", attr["cn"])
        attr_elems.appendChild(node)

        value = doc.createCDATASection(attr["val"])
        node.appendChild(value)

        attr_elems.appendChild(node)
def get_list(sum_barrier, sum_barrier_CN, sum_val):
    attr_list = []
    for i in range(len(sum_barrier)):
        temp_map = {}
        temp_map['en'] = sum_barrier[i]
        temp_map['cn'] = sum_barrier_CN[i].encode('utf-8')
        temp_map['val'] = sum_val[i]
        attr_list.append(temp_map)
    return attr_list

def dump2filesum(filename, summary, deadline):
    global ipaddress
    startime = get_new_time(deadline, -time_interval)
    endtime = deadline
    suc_user = summary["success"]
    suc_sys = summary["total"] - summary["servfail"]
    
    non_fail = summary["success"] + summary["nxdomain"]
    fail_num = summary["total"] - non_fail
    frequency = 1.0 * summary["total"] / (time_interval)
    ratio_sys = 100.0 * suc_sys / summary["total"]
    ratio_user =100.0 * suc_user / summary["total"]
    delay_avg = 1.0 * summary["delay-average"] / summary["total"]

    line2write = [startime, endtime, ipaddress, str(suc_user), str(suc_sys), str(fail_num), str(summary["recursion"]), str(summary["total"]-summary["recursion"]), str(summary["total"]), str(frequency), str(ratio_sys), str(ratio_user), str(delay_avg)]

    attr_list = get_list(sum_barrier, sum_barrier_cn, line2write)

    doc, elem = gen_doc_and_class('DNS business statistics', "DNS业务量统计报表")
    add_attr_value(doc, elem, attr_list)
    global file_no
    filename +=  '_' + startime + '_' + str(file_no["summary"]) + '.xml'
    f = file(filename, "w")
    doc.writexml(f, indent = ' ', addindent ='\t', newl = '\n', encoding="utf-8")
    f.close()
    file_no["summary"] += 1
    pack_and_upload(filename)
def pack_and_upload(filename):
    path = os.path.dirname(filename)
    name = os.path.basename(filename)
    dirname = filename[:filename.find('.xml')]
    os.system("mkdir -p %s " % dirname) 
    os.system("mv %s %s" % (filename, dirname))
    cur_path = os.getcwd()
    os.chdir(path)
    cmd = 'tar cvf ' + name +'.gz' + ' ' + name[0:name.find('.xml')]
    os.system(cmd)
    os.system("rm -rf " + dirname)
    os.chdir(cur_path)
    os.system(cur_path + "/upload.sh")

def dump2filetop(filename, dicttop, deadline):
    global ipaddress
    total = 0
    for k, v in dicttop.iteritems():
        total += v["count"]

    listst = sorted(dicttop.iteritems(),
                  key = lambda x: x[1]["count"], reverse = True)[0:top_n]

    startime = get_new_time(deadline, -time_interval)
    endtime = deadline
    doc, elem = gen_doc_and_class('Domain name system request ranking statistics', "域名解析请求排行统计指标组")
    for k, v in listst:
        amount_ratio = 100.0*v["count"] / total
        succ_num = v["success"] + v["nxdomain"]
        suc_ratio = 100.0*(succ_num) / v["count"]
        line2write = [startime, endtime, ipaddress, k, str(v["count"]), str(total),
                    str(amount_ratio), str(suc_ratio)]
        attr_list = get_list(top_barrier, top_barrier_cn, line2write)

        add_attr_value(doc, elem, attr_list)
    global file_no
    filename +=  '_' + startime + '_' + str(file_no["top"]) + '.xml'
    f = file(filename, "w")
    doc.writexml(f, indent = ' ', addindent ='\t', newl = '\n', encoding="utf-8")
    f.close()
    file_no["top"] += 1
    pack_and_upload(filename)

def dump2fileimp(filename, dictimp, deadline):
    global ipaddress
    startime = get_new_time(deadline, -time_interval)
    endtime = deadline
    doc, elem = gen_doc_and_class('DNS important domain name statistics', "DNS重点域名解析统计报表")
    for k, v in dictimp.iteritems():
        succ_num = v["success"] + v["nxdomain"]
        fail_num = v["count"] - succ_num
        frequency = 1.0 * v["count"] / (time_interval)
        ratio = 100.0 * (v["count"] - v["servfail"]) / v["count"]
        delay = 1.0* v["delay"] / v["count"]

        line2write = [startime, endtime, ipaddress, k,
                    str(v["count"]), str(succ_num), str(fail_num),
                    str(frequency), str(ratio), str(delay)]
        attr_list = get_list(imp_barrier, imp_barrier_cn, line2write)

        add_attr_value(doc, elem, attr_list)
    global file_no
    filename +=  '_' + startime + '_' + str(file_no["imp"]) + '.xml'
    f = file(filename, "w")
    doc.writexml(f, indent = ' ', addindent ='\t', newl = '\n', encoding="utf-8")
    f.close()
    file_no["imp"] += 1
    pack_and_upload(filename)

def cleandomain(dn):
    '''
    translate domain from "www.baidu.com" to "baidu.com", or
    "www.sina.com.cn" to "sina.com.cn"
    '''
    li = dn.split('.')
    d = '.'.join(li[-2:])
    if(d in topwords):
        d='.'.join(li[-3:])

    return d
def get_imp_domain(domain):
    imp_domain = None
    for imp_domain in important_domain: 
        if imp_domain[0] == "*":
            match_domain = imp_domain[1:]
        else:
            match_domain = imp_domain
        if domain.rfind(match_domain) != -1:
            break
    return imp_domain
def update_statistic(domainname, errorcode, delay, summary, dicttop, dictimp, reprog):
    summary["total"] += 1
    if delay != 0:
        summary["recursion"] += 1        # query is recursion
    summary["delay-average"] += delay
    if errorcode  == 0:
        summary["success"] += 1
    elif errorcode == 1:           # user err
        summary["user-err"] += 1
    elif errorcode == 2:
        summary["servfail"] += 1
    elif errorcode == 3:
        summary["nxdomain"] += 1
    elif errorcode == 4:
        summary["non-imp"] += 1
    elif errorcode == 5:
        summary["refuse"] += 1
    else:
        sys.exit(1)

    clndomainname = cleandomain(domainname)
    dicttop.setdefault(clndomainname, {"count":0, "success":0, "user-err":0, "nxdomain":0, "servfail":0, "non-imp":0, "refuse":0})
    dicttop[clndomainname]["count"] += 1
    if (errorcode == 0):
        dicttop[clndomainname]["success"] += 1
    elif(errorcode == 1):
        dicttop[clndomainname]["user-err"] += 1
    elif (errorcode == 2):
        dicttop[clndomainname]["servfail"] += 1
    elif (errorcode == 3):
        dicttop[clndomainname]["nxdomain"] += 1
    elif (errorcode == 4):
        dicttop[clndomainname]["non-imp"] += 1
    elif (errorcode == 5):
        dicttop[clndomainname]["refuse"] += 1
    else:
        sys.exit(1)

    val = reprog.match(domainname)
    if val:
        domain = val.group(0)
        imp_domain = get_imp_domain(domain)

        dictimp.setdefault(imp_domain, {"count":0, "success":0, "user-err":0, "servfail":0, "nxdomain":0, "non-imp":0, "refuse":0, "delay":0})
        dictimp[imp_domain]["count"] += 1
        dictimp[imp_domain]["delay"] += delay
        if(errorcode == 0):
            dictimp[imp_domain]["success"] += 1
        elif(errorcode == 1):
            dictimp[imp_domain]["user-err"] += 1
        elif(errorcode == 2):
            dictimp[imp_domain]["servfail"] += 1
        elif(errorcode == 3):
            dictimp[imp_domain]["nxdomain"] += 1
        elif(errorcode == 4):
            dictimp[imp_domain]["non-imp"] += 1
        elif(errorcode == 5):
            dictimp[imp_domain]["refuse"] += 1

def parseline(line, deadline, summary, dicttop, filename, reprog, dictimp):
    words = line.strip().split('|')
    if len(words) < 6:
        return deadline
    t = words[2]
    errorcode = int(words[4])
    delay = int(words[5])
    if words[1][-1] == '.':
        words[1] = words[1][0:-1]

    domainname = words[1]
    # the period is over, dump the result to file and start a new one
    if t >= deadline:
        if summary["total"] != 0:
            if is_file_split == "yes":
                sub_file = filename + ".summary"
                dump2filesum(sub_file, summary, deadline)
            else:
                dump2filesum(filename, summary, deadline)
            summary_init(summary)

        if len(dictimp) != 0:
            if is_file_split == "yes":
                sub_file = filename + ".important"
                dump2fileimp(sub_file, dictimp, deadline)
            else:
                dump2fileimp(filename, dictimp, deadline)
            dictimp.clear()

        if len(dicttop) != 0:
            if is_file_split == "yes":
                sub_file = filename + ".top"
                dump2filetop(sub_file, dicttop, deadline)
            else:
                dump2filetop(filename, dicttop, deadline)
            dicttop.clear()
        t = get_new_time(t, time_interval)
    else:
        t = None

    update_statistic(domainname, errorcode, delay, summary, dicttop, dictimp, reprog)

    return t

def get_ip_address(ifname):
    import socket
    import fcntl
    import struct

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
            s.fileno(),
            0x8915,  # SIOCGIFADDR
            struct.pack('256s', ifname[:15])
            )[20:24])

def summary_init(summary):
    summary["total"] = 0
    summary["success"] = 0
    summary["user-err"] = 0
    summary["recursion"] = 0
    summary["servfail"] = 0
    summary["nxdomain"] = 0
    summary["non-imp"] = 0
    summary["refuse"] = 0
    summary["delay-average"] = 0
    

if __name__ == "__main__":
    filename=sys.argv[1]
    deadline = '20100301100000'
    dicttop = {}
    dictimp = {}
    summary = {}
    summary_init(summary)
    reprog = re.compile('|'.join(str2restr(important_domain)))

    global ipaddress
    ipaddress = get_ip_address(eth_device)

    while True:
        line = getline()
        #print line
        ndl = parseline(line, deadline, summary, dicttop, filename, reprog, dictimp)
        if (ndl):
            deadline = ndl


